{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup as bs\n",
    "import requests\n",
    "import os\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "Books = pd.DataFrame({'Name': [\"XiYouJi\", \"HongLouMeng\", \"ShuiHuZhuan\", \"SanGuoYanYi\"], \n",
    "                     'url':  [\"https://xiyouji.5000yan.com/\", \"https://hongloumeng.5000yan.com/\", \"https://shuihu.5000yan.com/\", \"https://sanguo.5000yan.com/\"]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'NoneType' object has no attribute 'ul'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[7], line 8\u001b[0m\n\u001b[0;32m      6\u001b[0m response \u001b[38;5;241m=\u001b[39m requests\u001b[38;5;241m.\u001b[39mget(url)\n\u001b[0;32m      7\u001b[0m soup \u001b[38;5;241m=\u001b[39m bs(response\u001b[38;5;241m.\u001b[39mcontent)\n\u001b[1;32m----> 8\u001b[0m contents \u001b[38;5;241m=\u001b[39m soup\u001b[38;5;241m.\u001b[39mfind(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdiv\u001b[39m\u001b[38;5;124m'\u001b[39m,class_\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mp-2 my-2 bg-white rounded\u001b[39m\u001b[38;5;124m'\u001b[39m)\u001b[38;5;241m.\u001b[39mul\u001b[38;5;241m.\u001b[39mchildren\n\u001b[0;32m      9\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m contents \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m     10\u001b[0m     contents \u001b[38;5;241m=\u001b[39m soup\u001b[38;5;241m.\u001b[39mfind(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdiv\u001b[39m\u001b[38;5;124m'\u001b[39m,class_\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mp-3 my-2 bg-white rounded\u001b[39m\u001b[38;5;124m'\u001b[39m)\u001b[38;5;241m.\u001b[39mul\u001b[38;5;241m.\u001b[39mchildren\n",
      "\u001b[1;31mAttributeError\u001b[0m: 'NoneType' object has no attribute 'ul'"
     ]
    }
   ],
   "source": [
    "# 爬取西游记、红楼梦、水浒传、三国演义\n",
    "\n",
    "for idx, row in Books.iterrows():\n",
    "    book_name = row['Name']\n",
    "    url = row['url']\n",
    "    response = requests.get(url)\n",
    "    soup = bs(response.content)\n",
    "    contents = soup.find('div',class_='p-2 my-2 bg-white rounded')\n",
    "    if contents is None:\n",
    "        contents = soup.find('div',class_='p-3 my-2 bg-white rounded')\n",
    "\n",
    "    contents = contents.ul.children\n",
    "    for chapter in contents:\n",
    "        if chapter=='\\n' or chapter is None:\n",
    "            continue\n",
    "        # print(chapter)\n",
    "        chapter_name = chapter.a.text.replace('?','')\n",
    "        chapter_link = chapter.a.get('href')\n",
    "        # 获取章节内容\n",
    "        chapter_response = requests.get(chapter_link)\n",
    "        chapter_soup = bs(chapter_response.content)\n",
    "        chapter_content = chapter_soup.find('div',class_='py-2 px-2 px-md-4 bg-white divider')\n",
    "        # 获取文本，并去除空行\n",
    "        if chapter_content is None or chapter_content=='\\n':\n",
    "            continue\n",
    "        chapter_text = chapter_content.text\n",
    "        lines = chapter_text.splitlines()\n",
    "        lines = [line for line in lines if line.strip()]# 移除字符串前后的指定字符(空格或换行)\n",
    "        chapter_text = '\\n'.join(lines)\n",
    "        # print(chapter_text)\n",
    "\n",
    "        # 文本保存\n",
    "        if not os.path.exists(\".\\\\data\\\\\"+book_name):\n",
    "            os.makedirs(\".\\\\data\\\\\"+book_name)\n",
    "        with open(\".\\\\data\\\\\"+book_name+\"\\\\\"+chapter_name+\".txt\", 'w', encoding='utf-8') as handler:\n",
    "            handler.write(chapter_text)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
